-- Copyright (c) 2025 Huawei Technologies Co., Ltd.
-- openUBMC is licensed under Mulan PSL v2.
-- You can use this software according to the terms and conditions of the Mulan PSL v2.
-- You may obtain a copy of Mulan PSL v2 at:
--         http://license.coscl.org.cn/MulanPSL2
-- THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
-- EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
-- MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
-- See the Mulan PSL v2 for more details.
--
-- Create: 2025-5-9
-- Description: management status action.

local log = require 'mc.logging'
local class = require 'mc.class'
local skynet = require 'skynet'
local file = require 'utils.file'
local custom_messages = require 'messages.custom'
local vos = require 'utils.vos'
local context = require 'mc.context'
local client_ok, cnc_client_instance = pcall(require, 'mc.cnc.client')
local service_ok, cnc_server_instance = pcall(require, 'mc.cnc.server')
local const = require 'constants'
local comm = require 'common'
local is_first_startup = true

local mgmt = class()

-- 远程方法名常量定义
local REMOTE_METHOD_GET_BASIC_STATUS = 'get_basic_status'
local REMOTE_METHOD_GET_FAILOVER_RESTRAINT = 'get_failover_restraint'

-- 远端节点状态检查处理器（责任链模式）
-- 每个处理器检查一种状态，异常时返回相应错误码，成功返回 const.FAILOVER_CHECK_SUCCESS
local remote_check_handlers = {
    -- 基本状态检查（包含健康状态）
    base_state_check = function(self, name)
        -- 获取基本状态（包含健康状态）
        local basic_rsp = self:invoke_remote_method(name, REMOTE_METHOD_GET_BASIC_STATUS)
        if not basic_rsp then
            log:error('get remote [%s] basic properties failed, network communication error', name)
            return const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT
        end

        -- 解包 get_basic_status 的返回值
        -- CNC 返回的表是数组形式：{ActiveState, Health, HeartBeatState}
        local basic_props = {}
        if type(basic_rsp) == 'table' then
            basic_props.ActiveState = basic_rsp[1]
            basic_props.Health = basic_rsp[2]
            basic_props.HeartBeatState = basic_rsp[3]
        else
            log:error('get remote [%s] basic properties failed, unexpected response type', name)
            return const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT
        end

        local health_status = basic_props.Health
        if health_status == 'null' or health_status == const.BOARD_NOT_HEALTH then
            log:error('remote [%s] health status abnormal: %s', name, health_status)
            return const.FAILOVER_FAILED_REASON_REMOTE_NOT_HEALTH
        elseif health_status == const.BOARD_HEALTH_SIGNAL_FAULT then
            log:warn('remote [%s] health signal fault: %s', name, health_status)
            return const.FAILOVER_FAILED_REASON_REMOTE_NOT_HEALTH
        end

        log:info('remote [%s] health status check passed: %s', name, health_status)
        return const.FAILOVER_CHECK_SUCCESS
    end,

    -- 抑制状态检查
    restraint_state_check = function(self, name)
        -- 获取抑制状态
        local restraint_rsp = self:invoke_remote_method(name, REMOTE_METHOD_GET_FAILOVER_RESTRAINT)
        if not restraint_rsp then
            log:error('get remote [%s] restraint properties failed, network communication error', name)
            return const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT
        end

        -- 解包 get_failover_restraint 的返回值
        local failover_restraint
        if type(restraint_rsp) == 'table' then
            failover_restraint = restraint_rsp.FailoverRestraint or restraint_rsp[1]
        else
            log:error('get remote [%s] restraint properties failed, unexpected response type', name)
            return const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT
        end

        if failover_restraint == true then
            log:error('remote [%s] is in failover restraint state', name)
            return const.FAILOVER_FAILED_REASON_RESTRAIN
        end

        log:info('remote [%s] restraint state check passed', name)
        return const.FAILOVER_CHECK_SUCCESS
    end
}

function mgmt:ctor(obj_mgmt, sigs)
    self.obj_mgmt = obj_mgmt
    self.sigs = sigs
    self.active_state = const.STANDBY_STATE
    self.heartbeat_info = {
        heartbeat_count = const.DEFAULT_SOFT_HEARTBEAT_COUNTDOWN
    }
    self.failover_info = {
        last_failover_time = 0,  -- 上一次倒换时间，默认值0表示没有倒换过
        allow_failover = true,  -- 是否允许倒换
        failover_request_reason = const.FAILOVER_OCCURRED_BY_INIT_STATE,  -- 倒换请求原因
        failover_request_action = nil  -- 倒换请求方向
    }
    self.cnc_service = nil  -- 本端BMC创建的服务端，用于支持其他节点访问
    self.cnc_client_list = {}  -- 储存用访问其他节点的客户端
end

local failover_result_msg = {
    [const.FAILOVER_FAILED_REASON_REMOTE_NOT_HEALTH] = 'Remote smm cpld hearbeat abnormal',
    [const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT] = 'Network to remote smm disconnected',
    [const.FAILOVER_FAILED_REASON_LOCAL_CRITICAL_EVENT] = 'Local smm abnormal',
    [const.FAILOVER_FAILED_REASON_RESTRAIN] = 'Failover temporarily not allow. Please wait for a second',
    [const.FAILOVER_FAILED_REASON_BOARD_IN_NOT_OK] = 'Local board not in its position',
    [const.FAILOVER_FAILED_REASON_LOCAL_HEATBEAT_ABNORMAL] = 'Local smm cpld heartbeat abnormal',
    [const.FAILOVER_FAILED_REASON_OTHER] = 'other reason'  -- 实际上无此响应信息，对外接口不会触发此原因
}

function mgmt:initialize()
    -- 初始化信号监听
    self.sigs.start:on(function()
        -- fork操作避免影响对象注册
        skynet.fork_once(function()
            self:start_heartbeat_task()
            self:start_as_status_check_task()
            self:start_failover_task()
            self:create_cnc_service()
        end)
    end)
    self.sigs.failover:on(function(direction)
        local check_res, msg = self:check_failover_condition(direction)
        if check_res ~= const.FAILOVER_CHECK_SUCCESS then
            msg = check_res == const.FAILOVER_FAILED_REASON_BUTT and msg or failover_result_msg[check_res]
            log:error('start failover by switch rule failed [%s]', msg)
            return
        end
        self.failover_info.failover_request_reason = const.FAILOVER_OCCURRED_BY_SWITCH_RULE
        self.failover_info.failover_request_action = direction
    end)
    self.sigs.restart_service:on(function()
        if self.cnc_service then
            self.cnc_server:destroy()
            self.cnc_server = nil
        end
        self:create_cnc_service()
    end)
    self.sigs.restart_client:on(function(object_name)
        if self.cnc_client_list[object_name] then
            self.cnc_client_list[object_name]:destroy()
            self.cnc_client_list[object_name] = nil
        end
        self:create_cnc_client(object_name)
    end)
end

local function switch_status_op(ctx, is_success, msg)
    if is_success then
        log:operation(ctx, 'active_standby_mgmt', 'Set failover Success.')
    else
        log:operation(ctx, 'active_standby_mgmt', string.format('Set failover failed. %s', msg))
    end
end

function mgmt:switch_status(ctx, status)
    log:notice('start switch local status to [%s]', status)
    -- 对外接口会进行校验，这里增加防护
    if status ~= 0 and status ~= 1 then
        log:error('switch status [%s] is invalid', status)
        switch_status_op(ctx, false, failover_result_msg[const.FAILOVER_FAILED_REASON_OTHER])
        error(custom_messages.PropertyValueError('Status'))
    end
    local check_result, msg = self:check_failover_condition(status)
    if check_result ~= const.FAILOVER_CHECK_SUCCESS then
        msg = check_result == const.FAILOVER_FAILED_REASON_BUTT and msg or failover_result_msg[check_result]
        switch_status_op(ctx, false, msg)
        error(custom_messages.ForceFailoverError(msg))
    end
    -- 只要检测允许倒换即认为接口调用成功，不关注实际结果
    self.failover_info.failover_request_reason = const.FAILOVER_OCCURRED_BY_RPC
    self.failover_info.failover_request_action = status
    switch_status_op(ctx, true)
end

function mgmt:start_heartbeat_task()
    log:notice('heartbeat task start')
    -- 单位ms，skynet定时器精度10ms，使用时需要转换
    local delay = self.obj_mgmt.as_service.HeartbeatIntervalMS / 10
    delay = delay < 1 and 1 or delay  -- 最小值10ms
    local timeout = self.obj_mgmt.as_service.HeartbeatTimeoutMS
    timeout = timeout < 500 and 500 or timeout  -- 最小值500ms
    skynet.fork_once(function()
        comm.pcall_write(self.obj_mgmt.as_local, 'HeartbeatWrite', 1)
        comm.pcall_write(self.obj_mgmt.as_local, 'HeartbeatWrite', 0)
        local heartbeat_status = 1  -- 心跳状态默认正常（1），频繁访问的资源优先通过变量维护
        local pre_tick, cur_tick = 0, 0  -- 用于计算心跳间隔，初始值0不参与计算
        while true do
            skynet.sleep(delay)
            -- 心跳中断时检测是否有重新喂狗
            if heartbeat_status == 0 and self.heartbeat_info.heartbeat_count == 0 then
                goto continue
            end
            -- 判断2次心跳间隔是否超时
            cur_tick = vos.vos_tick_get()
            if (cur_tick - pre_tick) > timeout and pre_tick ~= 0 then
                log:warn('actual heartbeat interval [%s] timeout[%s]', cur_tick - pre_tick, timeout)
            end
            pre_tick = vos.vos_tick_get()
            -- 判断心跳计数
            if self.heartbeat_info.heartbeat_count ~= 0 then
                -- 心跳重新喂狗
                if heartbeat_status == 0 then
                    heartbeat_status = 1
                    self.obj_mgmt.as_local.heartbeat_status = heartbeat_status
                    log:notice('heartbeat start again')
                end
                self.heartbeat_info.heartbeat_count = self.heartbeat_info.heartbeat_count - 1
            else
                log:warn('heartbeat stopped')
                log:running(log.RLOG_WARM, 'active standby heartbeat interrupt')
                heartbeat_status = 0
                self.obj_mgmt.as_local.heartbeat_status = heartbeat_status
            end
            -- 发送心跳
            if heartbeat_status == 1 then
                comm.pcall_write(self.obj_mgmt.as_local, 'HeartbeatWrite', 1)
                comm.pcall_write(self.obj_mgmt.as_local, 'HeartbeatWrite', 0)
            end
            ::continue::
        end
    end)
end

-- 检测单板是否插稳
local function check_board_in(as_service, board_in_check_res, detection_state)
    log:info('checking borad in status')
    local board_in
    local count = 10 -- 连续读10次
    local success_count = 0 -- 成功次数，从0计数
    while count ~= 0 do
        board_in = comm.pcall_read(as_service, 'BoardFullySeated')

        -- 读失败不清空计数
        if not board_in then
            board_in_check_res.board_in = const.BOARD_IN_NOT_OK
        elseif board_in == const.BOARD_IN_OK then
            success_count = success_count + 1
        elseif board_in == const.BOARD_IN_NOT_OK then
            success_count = 0
        end

        if success_count == 3 then  -- 连续读到3次成功即可
            board_in_check_res.board_in = const.BOARD_IN_OK
            break
        end
        count = count - 1
    end
    -- 更新插稳状态检测结果
    if board_in_check_res.board_in == const.BOARD_IN_OK then
        log:notice('board in status check ok')
        detection_state.state = const.AS_CHECK_STATE_ACTIVE_CHECK
    else
        --  连续检测失败打印日志
        board_in_check_res.board_in_counter = board_in_check_res.board_in_counter + 1
        if (board_in_check_res.board_in_counter % const.BOARD_IN_COUNT_THRESHOLD) == 0 then
            log:error('board in status check failed')
        end
    end
end

-- 检测主备信号
function mgmt:check_as_active_state()
    log:info('checking active standby state')
    local local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')
    if not local_state then
        log:error('get local active standby status failed')
        return
    end

    -- 判断远端是否在位
    local is_remote_present = false
    for _, obj in pairs(self.obj_mgmt.as_remote_list) do
        if obj.mdb_obj.Presence == const.BOARD_PRESENT then
            is_remote_present = true
            break
        end
    end
    -- 远端都不在位
    if not is_remote_present then
        log:info('no remote board present, forcing local_state to ACTIVE')
        local_state = const.ACTIVE_STATE
    end
    -- 主备状态变更
    if self.active_state ~= local_state then
        log:notice('detected active standby state change: current=%s, new=%s', self.active_state, local_state)

        -- 在状态变更前进行异常信号检查
        local before_check_result = self:check_before_status_change()
        if before_check_result ~= const.FAILOVER_CHECK_SUCCESS then
            log:error('active status change to %s failed, reason: %s', local_state == const.ACTIVE_STATE and 'Active' or 'Standby', before_check_result)
            -- 检查失败时阻止状态变更，保持当前状态
            return
        end
        log:notice('active status change to %s check passed, proceeding', local_state == const.ACTIVE_STATE and 'Active' or 'Standby')

        if local_state == const.ACTIVE_STATE then
            log:notice('active standby status change to Active')
            -- 升主后开启15s的倒换抑制
            self.failover_info.last_failover_time = vos.vos_tick_get()
            self.failover_info.allow_failover = false
            self.obj_mgmt.as_local.mdb_obj.ActiveStatus = const.ACTIVE_STATE
            self.active_state = const.ACTIVE_STATE
            self:update_remote_props(const.STANDBY_STATE)
        elseif local_state == const.STANDBY_STATE then
            log:notice('active standby status change to Standby')
            self.obj_mgmt.as_local.mdb_obj.ActiveStatus = const.STANDBY_STATE
            self.active_state = const.STANDBY_STATE
            self:update_remote_props(const.ACTIVE_STATE)
        else
            log:info('undefined active standby status [%s]', local_state)  -- 避免刷日志
        end
    end
end

-- 检测主备状态
function mgmt:check_as_status(detection_state, as_status_check_param)
    log:info('checking active standby status')
    -- 判断心跳够1s才开启检测
    if (const.TICK_ONE_SECOND / as_status_check_param.heartbeat_interval) > as_status_check_param.active_counter then
        self.heartbeat_info.heartbeat_count = const.DEFAULT_SOFT_HEARTBEAT_COUNTDOWN  -- 心跳喂狗
        as_status_check_param.active_counter = as_status_check_param.active_counter + 1
        return
    end
    self:check_as_active_state()
    -- 更新当前检测阶段
    if self.active_state == const.ACTIVE_STATE then
        detection_state.state = const.AS_CHECK_STATE_IN_ACTIVE
    elseif self.active_state == const.STANDBY_STATE then
        detection_state.state = const.AS_CHECK_STATE_IN_STANDBY
    end
end

function mgmt:start_as_status_check_task()
    -- 非默认单板延迟5s启动
    if self.obj_mgmt.as_local.id ~= self.obj_mgmt.as_service.DefaultActiveId then
        skynet.sleep(500)
    end

    log:notice('active standby status task start')
    -- 单位ms，skynet定时器精度10ms，使用时需要转换
    local delay = self.obj_mgmt.as_service.ActiveMonitorIntervalMS / 10
    delay = delay < 5 and 5 or delay  -- 最小值50ms(计算结果为5)
    -- 当前检测阶段
    local detection_state = {state = const.AS_CHECK_STATE_WAIT_BOARD_IN}
    -- 插稳状态检测次数/结果
    local board_in_check_res = {board_in_counter = 0, board_in = const.BOARD_IN_NOT_OK}
    -- 主备状态检测参数，包含抢主计数器/心跳间隔
    local as_status_check_param = {
        active_counter = 1,
        heartbeat_interval = self.obj_mgmt.as_service.HeartbeatIntervalMS < 10 and 10 or  -- 最小值10ms
            self.obj_mgmt.as_service.HeartbeatIntervalMS
    }
    skynet.fork_once(function()
        while true do
            skynet.sleep(delay)
            if detection_state.state == const.AS_CHECK_STATE_WAIT_BOARD_IN then
                check_board_in(self.obj_mgmt.as_service, board_in_check_res, detection_state)
            elseif detection_state.state == const.AS_CHECK_STATE_ACTIVE_CHECK then  -- 开始检测主备状态
                self:check_as_status(detection_state, as_status_check_param)
            else
                self.heartbeat_info.heartbeat_count = const.DEFAULT_SOFT_HEARTBEAT_COUNTDOWN  -- 维持心跳
                self:check_as_active_state()
            end
        end
    end)
end

-- 判断本端是否满足倒换条件
function mgmt:check_local_failover_condition()
    -- 是否插稳
    local board_in = comm.pcall_read(self.obj_mgmt.as_service, 'BoardFullySeated')
    if not board_in then
        log:error('get board in status failed')
        return const.FAILOVER_FAILED_REASON_LOCAL_CRITICAL_EVENT
    end
    if board_in ~= const.BOARD_IN_OK then
        log:error('board in status not ok')
        return const.FAILOVER_FAILED_REASON_BOARD_IN_NOT_OK
    end
    -- 查看健康状态
    local health = comm.pcall_read(self.obj_mgmt.as_service, 'HealthStatus')
    if not health then
        log:error('get board health status failed')
        return const.FAILOVER_FAILED_REASON_LOCAL_CRITICAL_EVENT
    end
    if health ~= const.BOARD_HEALTH then
        log:error('board health status not ok')
        return const.FAILOVER_FAILED_REASON_LOCAL_HEATBEAT_ABNORMAL
    end
    -- 检查是否存在禁止条件，不涉及访问资源树所以这里直接选择遍历即可
    for _, obj in pairs(self.obj_mgmt.switch_rule_list) do
        if obj.rule_type == const.SWITCH_RULE_TYPE_NOT_ALLOW and obj.state then
            log:error('failover failed because of switch rule [%s] assert', obj.name)
            return const.FAILOVER_FAILED_REASON_BUTT, obj.description
        end
    end
    return const.FAILOVER_CHECK_SUCCESS
end

-- 检查单个远端节点的健康状态和抑制状态
-- name: 远端节点名称
-- 返回码，const.FAILOVER_CHECK_SUCCESS 表示检查通过，其他值表示失败原因
-- 使用责任链模式，依次执行各个检查处理器
function mgmt:check_single_remote_health_and_restraint(name)
    -- 定义检查处理器的执行顺序
    local check_order = {'base_state_check', 'restraint_state_check'}

    -- 依次执行各个检查处理器
    local complete_code
    for _, handler_name in ipairs(check_order) do
        local handler_func = remote_check_handlers[handler_name]
        complete_code = handler_func(self, name)
        if complete_code ~= const.FAILOVER_CHECK_SUCCESS then
            return complete_code
        end
    end

    return const.FAILOVER_CHECK_SUCCESS
end

-- 判断远端是否满足倒换条件
function mgmt:check_remote_failover_condition()
    -- 查询是否有远端在位
    local remote_presence = {}
    for name, obj in pairs(self.obj_mgmt.as_remote_list) do
        if obj.mdb_obj.Presence == const.BOARD_PRESENT then
            remote_presence[name] = obj
        end
    end
    if not next(remote_presence) then
        log:error('not remote object presence')
        return const.FAILOVER_FAILED_REASON_LOCAL_CRITICAL_EVENT
    end
    -- 远端节点访问是否可以倒换(检查抑制条件/健康状态)
    for name, obj in pairs(remote_presence) do
        local check_result = self:check_single_remote_health_and_restraint(name)
        if check_result ~= const.FAILOVER_CHECK_SUCCESS then
            return check_result
        end
    end
    return const.FAILOVER_CHECK_SUCCESS
end

-- 主备倒换前的状态检查
function mgmt:check_before_status_change()
    -- 先获取远端节点的硬件主备状态，接着判断是否获取成功
    local remote_props = self:get_all_remote_props(REMOTE_METHOD_GET_BASIC_STATUS)
    if not remote_props or not next(remote_props) then
        log:error('get remote props failed or no remote nodes')
        return const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT
    end

    -- 检查是否成功获取到远端状态
    local remote_available = false
    for name, props in pairs(remote_props) do
        if props[1] and type(props[1]) == 'table' and props[1].ActiveState ~= 'null' then
            remote_available = true
            break
        end
    end

    if not remote_available then
        log:error('failed to get valid remote status')
        return const.FAILOVER_FAILED_REASON_REMOTE_NETWORK_DISCONNECT
    end

    -- 判断当前是不是服务首次启动
    if is_first_startup then
        local local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')
        if local_state then
            for name, props in pairs(remote_props) do
                if props[1] and type(props[1]) == 'table' and props[1].ActiveState == local_state then
                    log:warn('same hardware state detected at startup: local=%s, remote [%s]=%s', local_state, name, props[1].ActiveState)
                    -- 首次启动检测到双主或双备，直接返回错误，不允许状态变更
                    if local_state == const.ACTIVE_STATE then
                        log:error('switch failed, double active detected at startup [%s]', name)
                        is_first_startup = false
                        return const.FAILOVER_FAILED_REASON_RESTRAIN
                    elseif local_state == const.STANDBY_STATE then
                        log:error('switch failed, double standby detected at startup [%s]', name)
                        is_first_startup = false
                        return const.FAILOVER_FAILED_REASON_RESTRAIN
                    end
                end
            end
        end
        is_first_startup = false
    end

    -- 本端升主前检查到其他节点健康而本端不健康时，直接return退出
    if self.active_state == const.STANDBY_STATE then
        local local_health = comm.pcall_read(self.obj_mgmt.as_service, 'HealthStatus')
        if local_health == const.BOARD_NOT_HEALTH then
            -- 检查远端是否有健康的节点
            for name, props in pairs(remote_props) do
                if props[1] and type(props[1]) == 'table' and props[1].Health == const.BOARD_HEALTH then
                    log:error('local is unhealthy while remote [%s] is healthy, skip failover', name)
                    return const.FAILOVER_FAILED_REASON_LOCAL_HEATBEAT_ABNORMAL  -- 13表示心跳异常
                end
            end
        end
    end

    local local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')

    -- 根据“即将变更后的方向”做双主/双备抑制检查：
    -- check_before_status_change() 仅在检测到 self.active_state ~= local_state 时被调用，
    -- 因此应按 local_state（新状态）判断与远端是否构成双主/双备。
    if self.active_state ~= local_state then
        for name, props in pairs(remote_props) do
            if props[1] and type(props[1]) == 'table' and props[1].ActiveState == local_state then
                local state_name = local_state == const.ACTIVE_STATE and 'active' or 'standby'
                log:error('switch failed, double %s is not allowed [%s]', state_name, name)
                return const.FAILOVER_FAILED_REASON_RESTRAIN  -- 7表示倒换被抑制
            end
        end
    end
    return const.FAILOVER_CHECK_SUCCESS  -- 0表示检测允许倒换
end

-- 判断是否满足倒换条件
function mgmt:check_failover_condition(direction)
    local local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')
    if not local_state then
        log:error('get local active standby status failed')
        return const.FAILOVER_FAILED_REASON_LOCAL_CRITICAL_EVENT
    end
    -- 状态是否重复
    if direction == self.active_state then
        log:error('current active standby status repeat')
        -- 由于不再限制只能由主发起接口倒换，这里需要判断状态是否重复，统一返回此错误码
        return const.FAILOVER_FAILED_REASON_LOCAL_CRITICAL_EVENT
    end
    -- 已有倒换任务在处理中
    if self.failover_info.failover_request_reason ~= const.FAILOVER_OCCURRED_BY_INIT_STATE then
        log:error('a request currently being processed')
        return const.FAILOVER_FAILED_REASON_RESTRAIN
    end
    -- 检查是否处于升主后的抑制时间内
    if not self.failover_info.allow_failover then
        -- 已过抑制时间则还原标志
        if (vos.vos_tick_get() - self.failover_info.last_failover_time) >= 15 * const.TICK_ONE_SECOND then  -- 15s的抑制时间
            self.failover_info.allow_failover = true
            self.failover_info.last_failover_time = 0
        else
            log:error('failover not allow in 15 seconds')
            return const.FAILOVER_FAILED_REASON_RESTRAIN
        end
    end
    -- 是否信号已切换只是软件还未检测到
    if local_state ~= self.active_state then
        log:notice('stop failover, pending automatic update')
        return const.FAILOVER_FAILED_REASON_RESTRAIN
    end
    -- 检查本端远端是否允许倒换
    local res, msg = self:check_local_failover_condition()
    if res ~= const.FAILOVER_CHECK_SUCCESS then
        return res, msg
    end
    res, msg = self:check_remote_failover_condition()
    if res ~= const.FAILOVER_CHECK_SUCCESS then
        return res, msg
    end
    return res
end

function mgmt:write_active_to_standby()
    local ok = comm.pcall_write(self.obj_mgmt.as_service, 'ActiveToStandbyWriteProtect', 0)
    if not ok then
        log:error('do active to standby write protect close failed')
        return false
    end
    ok = comm.pcall_write(self.obj_mgmt.as_service, 'ActiveToStandbyWrite', 1)
    if not ok then
        log:error('do active to standby write failed')
        comm.pcall_write(self.obj_mgmt.as_service, 'ActiveToStandbyWriteProtect', 1)
        return false
    end
    ok = comm.pcall_write(self.obj_mgmt.as_service, 'ActiveToStandbyWriteProtect', 1)
    if not ok then
        log:error('do active to standby write protect open failed')
    end
    return true
end

function mgmt:write_standby_to_active()
    local ok = comm.pcall_write(self.obj_mgmt.as_service, 'StandbyToActiveWriteProtect', 0)
    if not ok then
        log:error('do active to standby write protect close failed')
        return false
    end
    ok = comm.pcall_write(self.obj_mgmt.as_service, 'StandbyToActiveWrite', 1)
    if not ok then
        log:error('do active to standby write failed')
        comm.pcall_write(self.obj_mgmt.as_service, 'StandbyToActiveWriteProtect', 1)
        return false
    end
    ok = comm.pcall_write(self.obj_mgmt.as_service, 'StandbyToActiveWriteProtect', 1)
    if not ok then
        log:error('do active to standby write protect open failed')
    end
    return true
end

function mgmt:set_standby_to_active()
    local ok = self:write_standby_to_active()
    if not ok then
        return false
    end
    -- 检查倒换结果是否成功
    local temp_tick = vos.vos_tick_get()
    local cur_tick = temp_tick
    local local_state
    while cur_tick - temp_tick < const.TICK_ONE_SECOND do
        skynet.sleep(20)  -- 抢主固定等待200ms(20)
        cur_tick = vos.vos_tick_get()
        local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')
        if not local_state then
            log:info('get local active standby status failed')
            goto continue
        end
        if local_state == const.ACTIVE_STATE then
            break
        end
        ::continue::
    end
    ok = comm.pcall_write(self.obj_mgmt.as_service, 'StandbyToActiveWrite', 0)
    if not ok then
        return false
    end
    if local_state ~= const.ACTIVE_STATE then
        return false
    end
    return true
end

function mgmt:set_active_to_standby()
    local ok = self:write_active_to_standby()
    if not ok then
        return false
    end
    -- 检查倒换结果是否成功
    local temp_tick = vos.vos_tick_get()
    local cur_tick = temp_tick
    local local_state
    while cur_tick - temp_tick < const.TICK_ONE_SECOND do
        skynet.sleep(1) -- 固定间隔10ms
        local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')
        if not local_state then
            log:info('get local active standby status failed')
            goto continue
        end
        -- 获取远端节点的实际主备状态
        -- 如果本端为备且远端有主才算成功
        cur_tick = vos.vos_tick_get()
        ::continue::
    end
    ok = comm.pcall_write(self.obj_mgmt.as_service, 'ActiveToStandbyWrite', 0)
    if not ok then
        return false
    end
    if local_state ~= const.STANDBY_STATE then
        return false
    end
    return true
end

function mgmt:start_failover_task()
    skynet.fork_once(function()
        while true do
            skynet.sleep(const.SKYNET_ONE_SECOND)
            if self.failover_info.failover_request_reason == const.FAILOVER_OCCURRED_BY_INIT_STATE then
                goto continue
            end
            local direction = self.failover_info.failover_request_action
            -- 表征此次请求已被处理，如果中途再有请求，此次处理会在check阶段被拦截
            self.failover_info.failover_request_reason = const.FAILOVER_OCCURRED_BY_INIT_STATE
            -- 主备倒换前的状态检查
            local before_check_result = self:check_before_status_change()
            if before_check_result ~= const.FAILOVER_CHECK_SUCCESS then
                log:error('check before status change failed, reason: %s', before_check_result)
                goto continue
            end
            -- 判断是否满足倒换条件
            local check_result, _ = self:check_failover_condition(direction)
            if check_result ~= const.FAILOVER_CHECK_SUCCESS then
                goto continue
            end
            local set_result = false
            -- 执行倒换操作
            if direction == const.ACTIVE_STATE then
                set_result = self:set_standby_to_active()
            else
                set_result = self:set_active_to_standby()
            end
            if set_result then
                log:notice('set active standby status success')
            else
                log:error('set active standby status failed')
            end
            ::continue::
        end
    end)
end

function mgmt:on_dump(_, path)
    log:notice('dump start')
    -- 校验路径
    if file.check_realpath_before_open_s(path) == -1 or file.check_shell_special_character_s(path) ~= 0 then
        log:error('[on_dump] path is invalid.')
        return
    end

    local info_file, err = file.open_s(path .. '/asm_info.txt', 'w')
    if not info_file then
        log:error('[dump] open file failed, err: %s', err)
        return
    end

    local res = {}
    local remote_props = self:get_all_remote_props(REMOTE_METHOD_GET_BASIC_STATUS)
    -- 主备状态
    local hw_local_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead')
    table.insert(res, string.format('local board as_status:%s\nlocal board hardware as_status:%s\n',
        self.active_state, hw_local_state))
    -- 本端心跳状态
    table.insert(res, string.format('heart beat status:%s\n', self.obj_mgmt.as_local.mdb_obj.heartbeat_status))
    -- 获取远端节点信息
    local remote_prop
    for name, obj in pairs(self.obj_mgmt.as_remote_list) do
        table.insert(res, string.format('remote board [name:%s id:%s] info:\n', name, obj.id))
        remote_prop = remote_props[name][1]
        if type(remote_prop) == 'table' then
            table.insert(res,
                string.format('as_status:%s\nremote board hardware as_status:%s\nheart beat status:%s\nhealth:%s\n',
                obj.mdb_obj.ActiveStatus, remote_prop.ActiveState, remote_prop.HeartBeatState, remote_prop.Health))
        else
            table.insert(res, string.format('as_status:%s\nremote board hardware as_status:null\nheart beat status:null\nhealth:null\n',
                obj.mdb_obj.ActiveStatus))
        end
    end
    -- 切换规则
    table.insert(res, string.format('%-15s | %-10s | %-10s | %-10s | %-10s | %-10s | %-10s | %-20s\n',
        'RuleName', 'RuleType', 'Enabled', 'Threshold', 'Reading', 'Operator', 'Direction', 'Description'))
    for name, obj in pairs(self.obj_mgmt.switch_rule_list) do
        table.insert(res, string.format('%-15s | %-10s | %-10s | %-10s | %-10s | %-10s | %-10s | %-20s\n',
            name, obj.rule_type, obj.mdb_obj.Enabled, obj.mdb_obj.Threshold, obj.mdb_obj.Reading,
                obj.operator, obj.direction, obj.description))
    end
    -- 心跳状态
    table.insert(res, string.format('heart beat status:%s\n', self.obj_mgmt.as_local.heartbeat_status))
    info_file:write(table.concat(res, ''))
    info_file:close()
    log:notice('dump end')
end

-- 更新远端资源树属性
function mgmt:update_remote_props(direction)
    -- 如果本端升主，意味着远端都为备
    if direction == const.STANDBY_STATE then
        for name, obj in pairs(self.obj_mgmt.as_remote_list) do
            obj.mdb_obj.ActiveStatus = const.STANDBY_STATE
        end
        return
    end
    -- 如果本端降备，则需要对信号切换后的远端进行升主
    local remote_props = self:get_all_remote_props(REMOTE_METHOD_GET_BASIC_STATUS)
    for name, obj in pairs(self.obj_mgmt.as_remote_list) do
        if remote_props[name][1] and type(remote_props[name][1]) == 'table' and remote_props[name][1].ActiveState == const.ACTIVE_STATE then
            obj.mdb_obj.ActiveStatus = const.ACTIVE_STATE
            return
        end
    end
end

--获取本端是否处于抑制状态
function mgmt:get_local_failover_restraint()
    local is_in_restraint = false
    local retry_count = 0
    local max_retries = 100
    local retry_interval = 5  -- 50ms转换为skynet时间单位(50ms/10ms=5)

    while retry_count < max_retries do
        if not self.failover_info.allow_failover then
            -- 已过抑制时间则还原标志
            if (vos.vos_tick_get() - self.failover_info.last_failover_time) >= 15 * const.TICK_ONE_SECOND then  -- 15s的抑制时间
                self.failover_info.allow_failover = true
                self.failover_info.last_failover_time = 0
                is_in_restraint = false
                break  -- 抑制状态已解除，退出重试
            else
                is_in_restraint = true
                retry_count = retry_count + 1
                -- 只在还有重试次数时才sleep
                if retry_count < max_retries then
                    skynet.sleep(retry_interval)  -- 等待50ms后重试
                end
            end
        else
            is_in_restraint = false
            break  -- 允许倒换，退出重试
        end
    end
    return is_in_restraint
end

-- 本端BMC需要创建一个服务端，响应远端节点的访问，即对于其他节点来说此BMC就是远端
function mgmt:create_cnc_service()
    if not service_ok then
        log:error('require setvice failed, err %s', cnc_server_instance)
        return
    end
    local ok, err
    local try_count = 0  --  统计重试次数（默认0）
    while not ok and try_count ~= 60 do  -- 重试60次（10分钟）
        ok, err = pcall(function()
            self.cnc_service = cnc_server_instance.new(self.obj_mgmt.as_local.service_addr)
        end)
        if not ok and (try_count % 10 == 0) then  -- 取10的余数，等于0打印日志，即每重试10次打印一次
            log:error('create cnc server failed, %s', err)
        end
        try_count = try_count + 1
        skynet.sleep(1000)  -- 间隔10秒重试，服务刚启动时可能失败（1000表征10秒）
    end

    log:notice('create cnc server success, try_count: [%s]', try_count)

    -- 定义不同method对应的处理方法（提取到回调函数外部，避免每次调用都重新创建）
    local method_handlers = {
        [REMOTE_METHOD_GET_BASIC_STATUS] = function()
            -- 功能1：只获取三种基本属性
            local active_state = comm.pcall_read(self.obj_mgmt.as_service, 'ActiveRead') or 'null'
            local health = comm.pcall_read(self.obj_mgmt.as_service, 'HealthStatus') or 'null'
            local heartbeat_state = self.obj_mgmt.as_local.heartbeat_status
            -- 返回多个值，而不是表
            return active_state, health, heartbeat_state
        end,
        [REMOTE_METHOD_GET_FAILOVER_RESTRAINT] = function()
            -- 功能2：获取抑制状态
            local res = {}
            res['FailoverRestraint'] = self:get_local_failover_restraint()
            return res
        end
    }

    -- 注册回调函数
    -- 根据method参数区分不同功能
    cnc_server_instance.on_call_remote_class_method(function(service_name, path, interface, method, signalure, ...)
        -- 根据method参数调用对应的处理方法
        if method and method_handlers[method] then
            local result = method_handlers[method]()
            return result
        else
            -- 无效的method参数，返回错误
            log:error('invalid method parameter: %s', method or 'nil')
            return nil
        end
    end)
    -- 当前暂无场景使用，预留接口
    cnc_server_instance.on_get_remote_property_value(function(service_name, path, interface_prop_list, ctx) return end)
end

-- 创建客户端服务，用于访问其他远端节点的回调函数
function mgmt:create_cnc_client(object_name)
    if not client_ok then
        log:error('require client failed, err %s', cnc_client_instance)
        return
    end
    local service_addr = self.obj_mgmt.as_remote_list[object_name].service_addr
    local ok, err = pcall(function()
        self.cnc_client_list[object_name] = cnc_client_instance.new(service_addr)
    end)
    if not ok then
        log:error('create cnc client to [%s] failed, %s', object_name, err)
        self.obj_mgmt.as_remote_list[object_name].mdb_obj.CommunicationStatus = 0
    else
        self.obj_mgmt.as_remote_list[object_name].mdb_obj.CommunicationStatus = 1
        log:notice('create cnc client to [%s] success', object_name)
    end
end

-- 定义远程方法的模板和填充回调
local remote_methods = {
    [REMOTE_METHOD_GET_BASIC_STATUS] = {
        default_object_props = {ActiveState = 'null', Health = 'null', HeartBeatState = 'null'},
        fill_object_props = function(rsp)
            local object_props = {}
            -- CNC 返回的表是数组形式：{ActiveState, Health, HeartBeatState}
            object_props.ActiveState = rsp[1]
            object_props.Health = rsp[2]
            object_props.HeartBeatState = rsp[3]
            return object_props
        end
    },
    [REMOTE_METHOD_GET_FAILOVER_RESTRAINT] = {
        default_object_props = {FailoverRestraint = 'null'},
        fill_object_props = function(rsp)
            local object_props = {}
            object_props.FailoverRestraint = rsp.FailoverRestraint or rsp[1]
            return object_props
        end
    }
}

-- 获取所有其他节点的属性
function mgmt:get_all_remote_props(method)
    local res = {}
    local method_config = remote_methods[method]

    -- 如果method不存在，返回空结果
    if not method_config then
        log:error('unsupported remote method: %s', method)
        return res
    end

    for name, obj in pairs(self.obj_mgmt.as_remote_list) do
        local rsp = self:invoke_remote_method(name, method)
        -- 获取属性失败或返回空表认为通信异常
        if not rsp or (type(rsp) == 'table' and not next(rsp)) then
            -- 使用模板中定义的默认值
            res[name] = {method_config.default_object_props}
            obj.mdb_obj.CommunicationStatus = 0
            goto continue
        end

        -- 使用模板中定义的填充回调解包返回值
        local object_props = {}
        if type(rsp) == 'table' then
            object_props = method_config.fill_object_props(rsp)
        else
            -- 非表类型，使用默认值
            log:warn('remote [%s] returned non-table response, using default values', name)
            object_props = method_config.default_object_props
        end

        res[name] = {object_props}  -- 保持原有的数组格式
        obj.mdb_obj.CommunicationStatus = 1
        ::continue::
    end
    return res
end

function mgmt:invoke_remote_method(object_name, method)
    if not self.cnc_client_list[object_name] then
        self:create_cnc_client(object_name)
    end
    if not self.cnc_client_list[object_name] then
        log:error('get prop of [%s] failed, no client object', object_name)
        return nil
    end
    -- 传入method参数到远程调用
    local ok, rsp = pcall(function()
        return self.cnc_client_list[object_name]:call_remote_class_method('bmc.kepler.active_standby_mgmt',
            '', '', method, '', {})
    end)
    if not ok then
        log:error('get prop of [%s] failed, [%s]', object_name, rsp)
        return nil
    end

    return rsp
end

return mgmt
